{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "from puckworld import PuckWorldEnv\n",
    "from agents import DQNAgent\n",
    "from utils import learning_curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "env = PuckWorldEnv() \n",
    "agent = DQNAgent(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/1000 [00:00<?, ?it/s]/home/qiang/Documents/Codes/c06/agents.py:222: UserWarning: invalid index of a 0-dim tensor. This will be an error in PyTorch 0.5. Use tensor.item() to convert a 0-dim tensor to a Python number\n",
      "  mean_loss = loss.sum().data[0] / self.batch_size\n",
      "100%|██████████| 1000/1000 [10:50<00:00,  1.54it/s]\n"
     ]
    }
   ],
   "source": [
    "data = agent.learning(gamma=0.99,          # 衰减引子\n",
    "               epsilon = 1,\n",
    "               decaying_epsilon = True,\n",
    "               alpha = 1e-3, \n",
    "               max_episode_num = 1000, \n",
    "               display = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No handles with labels found to put in legend.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZcAAAEWCAYAAACqitpwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvFvnyVgAAIABJREFUeJzsnXlcXNXd/9/fGWCAsARIgIQhYYhABBIiwRAiJi5tjXWtttXY2lhbW/t0/dXa2kfb2qfaPt1sa21dam2f1q1qa7XuK1k0iQYMERIhMSQGEiBhCZCwzpzfH/cOmRm2C4GRzL3v1+u+5t5zzj33fObee7737KKUwsLCwsLCYjKxfdgJsLCwsLAIPSzjYmFhYWEx6VjGxcLCwsJi0rGMi4WFhYXFpGMZFwsLCwuLSccyLhYWFhYWk45lXCwsDCAiOSKyTUQ6ReQbH3Z6LEBEykTkix/Stf8qIreN4q9E5JRgpmm6YRmXEEJE9opIt54BtovImyJyvYjYAsKtEJHX9HBHRORpEVno43+W/nL8MeC8jSJyTYCbN+z3plSc/zVvFZEHg3U9ne8CryulYpVSdwb52ic9+jNyVES6RKRBRO4QEfsUXWuNiOwMcHt5BLebpiINFpZxCUUuUkrFAvOB/wW+B/zZ6ykiJcBLwFPAXMAFbAfeEJEMn3iOAlcHuA3HWqAV+NykpH6aISJh+u58oPoE4zA7BUqpGOBc4Crguim6znpgoYjMhsH/vwCICnAr0cOOi6kyiqGGZVxCFKXUEaXU08AVwFoRyde9fgH8TSn1O6VUp1KqVSl1C/AW8COfKNqBvwa4+SEiM4BPAl8FskSkKMD/cyKyT0RaROQHesnqI7qfTURuEpH3df/HRCRR98vQv3TXisgHInJYRG7W/VYD/w1coX8FV46Qtr0i8n0R2SEibSLyFxGJ9PG/UK/m8pbwFgec+z0R2Q4cFZHXgLOBu/RrZotIvIj8TUQO6Rpv8ZYQReQaEXlDRH4jIi3ArQFu7SKyRy9BXiMi+0WkWUTW+qThAhF5R0Q6dP9bffxG/H90f7uI/Lf+33aKSLmIpOt+C/Uv9lYRqRGRT49yf+fqpdpWEdktItf5+N2q37O/6deoDrz/I6GUeg/YAOTrcflVIUlAlZOIXKLfqw5d0+ph0jpHRLaLyI1KqQZgD7BS9y5E+zBYF+BmA97Wzz9VtGq2dl3LxQHpuVtEnhORo2jPQuD1bxSRgyJyQESuNfI/hDxKKWsLkQ3YC3xkGPcPgK8A0YAbOHuYMJ8HGvT9s4B6IBXoAHJ0943ANT7nXA0cBOzAf4Df+/jlAl1AKRAB/Aro96YP+CawGXACDuBe4BHdLwNQwJ+AKLSvzl7gVN3/VuBBA/9FFZAOJAJvALfpfqcBzUCxnva1eniHz7nb9HOjdLcy4Is+8f8NrfQXq6e3FviC7ncNMAB8HQjTNXjdPq9f8zb9vvxB1/8xoBOI8bkHi9AywMVAE3Cpwf/nRuBdIAcQ3T8JmAHs19MQpv8Ph4HcEf7D9cAfgUhgCXAIOMfnHvQAH9f1/AzYPMr9UMApPs9Go8//NeinH//V514tA44AH9X/izRgoe89QSt91wJf8onjL8Dv9P3vAP+DVlLydXtN3w8HdqN9tEQA5+j3IscnPUeAM/Q0RAakcbV+f/L1//jhQE1m3D70BFjbJN7MkY3LZuBmtIxceV/OgDCrgT59/yygXt//BfAPfT/QuLwC/FbfX6NnPuH68Q/RjYV+HA30cdy47ATO9fGfg2Z8wjieeTp9/N8CrtT3b8WYcbne5/jjwPv6/t3ATwLC1wCrfM69NsC/DN24oGWmffhkysCXgTJ9/xrgg4DzrwF2+Rwv0jWm+Li1AEtG0PNb4Df6/lj/Tw1wyTBxXAFsCHC7F/jRMGHT0T5EYn3cfgb81ecevOLjlwt0j3I/FNqHShvwPppxtfn4jWRc7vXqHibOMuAO/X6tGeb/fkfffwrNOC0McPuRvn8mmrGz+Zz/CHCrT3r+FhC/bxofAP7Xxy87UJMZN6tazBykobWLtAEetIw8kDloX7GB/Bw4T0QKfB31apazgYd0p6fQvugu0I/non0lA6CUOoaWeXqZDzypV0O0oxkbN5DiE6bRZ/8YEDOyxGHZ77O/T0+T99o3eK+tXz/dxz/w3EBmoX3t7guIP22M85t89rsBlFKBbjEAIlIsIq/r1W5HgOv16/oy0v+TjpaBBzIfKA7Q/Rm0Emogc4FWpVTnKBoDrx8po7cvFSqlEpRSC5RStyilPKOE9TKSFi+fARqAJwLc1wOLRSQBWA5sUlp13BzdrZTj7S1zgf0B6TFyP73MZeizZnos4xLiiMjpaC/JRqXUUWAT8Klhgn4a7UvQD6VUC9pX808CvK5Ge37+IyKNaHXckWhVTKBVlzl90hGFVjXjZT9wvlJqps8WqbT68rEwOpV3us/+POCAz7VvD7h2tFLqEYPXOIxWypofEL9v2k90uvGHgaeBdKVUPHAPWhWXEfYDC0ZwXxegO0Yp9ZVhwh4AEkUk1sctUONkcQytZOvF19iNpMXLrWj342HxaWhXSu1B0/AltFJkl+61SXeLQSvRo4dLF/9eleO5nwcZ+qyZHsu4hCgiEiciFwKPolUhvat73YTWwP8NEYkVkQS98fRM4KcjRHcHsAI41cdtLfBjtLp473Y58HERSUL7krxIb7SOQMsEfDPHe4DbRWS+nt7ZInKJQXlNQEZAZjAcXxURp2gdBW4G/qG7/wm4Xi8diIjM0BvQY0eO6jhKKTfwmJ7+WF3Dt4HJ7B4di1Zy6BGRZWi9q4xyP/ATEcnS9S3W78kzQLaIXC0i4fp2uoicGhiBUmo/8CbwMxGJFK3DwxeYXI1etgFX6R0RVgOrfPz+DHxeRM4VrRNImvh0m0cz8p9Ca+v4W8AzsQHtvmzwcduou21VSnXrblvQDNx39f/kLOAitHfHCI8B14hIrohEM0onGDNhGZfQ4z8i0on2xXczmmH4vNdTKbUROA+4DO2LqxXNUJyrlKoaLkKlVAda24u3N9dytK/2PyilGn22p9EaRtcoparRGrQf1a/ThdaI3qtH+zu0L/OX9PRuRmtgN8Lj+m+LiFSMEu5htG7Xezhez49Saita4+5daFWFu9Hq6MfD19G6a+9By7AeRqt7nyz+C/gf/b/5IVoGZpQ79PAvobVz/BmtY0InWseBK9G+1hvRqj0dI8SzBq195wDwJFobxSvjVjI230TLzL3VdP/2eiil3kJ7fn+D1qi+Dv8SI0qpPrTnOQV4wMfArAOS0e6Plw262/qA8y8CzkcrBf0R+JxejTYmSqnn0Ur3r6E9S68ZOS/UEb0BysKk6F+krwNXKaVenMLrxKBlHllKqbqpuo7P9faiNcBPRWZoYWExBlbJxeQopbYDlwKLxmiMHTcicpGIRIs2HuZXaN1j907mNSwsLKYn1shhC5RSG/Cvl54sLgH+jtbWshWtq6xVVLawMAFWtZiFhYWFxaRjVYtZWFhYWEw6pq0WmzVrlsrIyPiwk2FhYWFxUlFeXn5YKTV7rHCmNS4ZGRls3brVcPj33nuPhQsXjh0whDCjZjCnbjNqBnPqPlHNImJoBgKrWswgjY2NYwcKMcyoGcyp24yawZy6g6XZMi4WFhYWFpOOZVwsLCwsLCYd07a5WFhYWJid/v5+6uvr6enpGeIXGRmJ0+kkPDx8QnFbxsUgZuxZZkbNYE7dZtQM5tTtq7m+vp7Y2FgyMjIQOT6vrFKKlpYW6uvrcblcE7pOyFSLichq0ZZt3S0iN012/ImJiZMd5bTHjJrBnLrNqBnMqdtXc09PD0lJSX6GBUBESEpKGrZEY5SQMC76Og5/QJvVNBdYIyK5k3mNpqamsQOFGGbUDObUbUbNYE7dgZoDDctY7kYJCeOCts72bqXUHn367EfR5rWaNJxO59iBQozhNHs8iqe2NXC0d+BDSFFw8OqubzvG/tZjQbmm26N4fOt+9hzqGuJ3qLOX1qN9E4p3d3Mnz717cMxwRp7vw129DDdd1IZdh2ho7x7mjKmnp989bJq8KKV4c/dhDnf1DvFr7uzhwMAM3tx9mI27DnPkWP9UJjVotB7t45ntB7jrtV08vOUDXt7RRG3T8QVFg5WXhUqbSxr+y4zWM8zaICJyK/pCPsnJyZSVlfn5p6am4nK5qK2tJS8vj/XrB5d8oKuri5iYGEpKSqirq8PpdNLU1MT+/f6rn6alpeF0OqmrqyM7O5uNGzcSSGlpKbW1tbhcLurr62loOL7gXa9bcd8OuPG8hcz0HMHlcrFp06YhcaxcuZLq6mqys7Opq6sb7Lt+tF970fKyXCQmJtLU1ITT6eQPT73BS/v6mREuxEYIydE2rr9wOUcP7iEvL4/a2lqam5v9rtHX18fpp59Oa2srKSkpbN26lS0HB7i7spfLssK5LCea0tJSKioqWLJkCdXV1bS0tPjFkZWVhcPhoKuri8TERN54q5yqw252tLjZ2eKmtVcRbrejlIdFzgS+UuCAng6/OHJycrDb7fT09BAbG0tlZSWgZciHuhWt/WHMmZ9Jan8jJcuKqKiooKPDP47c3Fxe2HGI9JkRLHImUFXlv3RNTEwMRUVFrN/0Nq/tbmdzo+K9Fi2zyU6wcWZaGNecsxibGsBms2G323nn3Wr+vqOP8qYB0mNtLJwdRUl+Jjt27yVq5mwOHjzIxS4h3Hb8C7CgoIDOzk4iIyNxu93U1NRwtF9x97Zeqlrc2AWuOcPFyqSjzM1cyI8f38ybDX14FMyKElzxNopznKREejh1bgKzo4W693f7aUlKSiIvL4+HXtrCLzZ10Nk7wJcWO1gx9/jrXlRUxMHGRlrdkWyobWZTzQF6CaNnQKEU3HLuXM4szKO6upqCggKeeP51blp/jMuzIzjfpTXwFhcX80bV+1z3z3044yO45fQwHGHHtY72PnkZ7/v01Mvr+UdNHwePKlq6PRwbgBkRdpKibMyfHUd+ko2F0UeJdwj7Otw8tLOP2jYPafGR/OTcZM4oyGbTpk3saHHz+3d66Pb5RpoTE8Zz3zyTpoYPhowFycjI8HuftmzZ4ucvIqxatYrKysoR36fMzExiY2P93idfwsLCBt+ngoICrrt/A/tau0meIaRE24i0C3EJSXiw0dPXS3zMDJobD+BW0Nmn6OxTHDwG+464h/zPADkJNi5aEE52nIclS5bQ09ODx+Ohs7PTL5yIEBMTw9GjRwGGfZ+MEBITV4rIJ4HVSqkv6sdXA8VKqa+NdE5RUZEazwj9srIyzjrrrBNN6phs3dvKJ+/ZxNXL5/OTS/MNn6eU4vGt9fzk2R0UOGfy4Bf9bevaB96iYl8baQlRtB7t41BXL0pBTkosnz49nWvPyBhSDA7U7PYoVv92Pbuau3DNmsFrN6waco5Sipd2NPHrl2pY5krk1ovyCLNrBeS9h4/ymfu30NDezYwIO8WZSZySHIPHo+hze3h8az1JMRH87dplZM6OIZCqhiM89+5Bdjd3sefwUfa1HKXfffz5/WhuCnd/pnDwer7saznKql+WYRP4TPF8vvOxHOKjj/eC2XOoi7+8sZcnyuvp7neTlRzDJwq1JdSf2FrPnsNHiY8K5wulLq45I4Mjx/q5/sFydhzs4IJFczjQ3k3VgQ76BrRl2B1hNnoHPHznY9l87Zysweu880Eb1/71bfLT4rlg0RyyUmL5zuOV1Lcd4/vnn8qu5i4effsDZkSE0dU7QFS4nauK55Ec62B7/REq69upbzteSkiNi2TtigyuWjbPT88LVQf5xqPbmJcYTXxUODsOdPDvr55BTmosSikeeWs/d7xcw+EurUQ0K0pwzoonxhHG5j0tfP6MDG6+4HjN8q9erOGu13cTHxXOhu+dTVykdq1vPPIOL1Y30uf2cOXp6fzsssVD/vvJorvPzeV3v8m+lqMsz0wiLSGK2TEOWo/1caC9mz2HjrKruQu7TchPi2d7fTsJ0RF8fkUGD7xRh91m42/XLuO9xg6++8R2FsyO4UJnH6cvPY2Gtm5ueLySL5S6+MGF/jXqHo+iruUo73zQTmS4jQsXzx0xjXsPH+X253ZyVfE8zs5JHjGcx6PY2djBlj2tvLO/nauWzaNkwfFVwP9ZXs8Nj1eSnxZHa1cfBzt68M2qRfA7jnWEkRgTQdrMKFYsSGLFKbM4NTWO9u4+DnX28lZdK/dvqKOxo4eMOBv3f7GUU5Jj2blzJwsXLhy2CkwpxXvvvcepp/ovVCoi5UqpohHF6YRKyaUB/zWsnUzNWt9TTo1efN24+/Co4Vq6etnbchSbCAMexe9f28362kNER9jZuq+VAbdnMJNVSlHVcITz8lP51acKAGju6OH5qkb+va2BnzyzgwWzZ3DWKC8DwHPvHmRXcxfnLEzmtfeaqfigjaXzjzcO7jjQwf88U83mPa3MiY/kwc0f0Hikh9+vKaShvZur/rSZfreHv39hGcszkwgPMAKXFzq59q9vc/ndb/LrTxfgTIgmzCbsbu7igTfq2LynlXC7MD9pBpmzZvCRU1NYMHsGC5JjqNjXxm3P7uS7/9zOrz5ZgM0WYChrDgFwyZI0Htqyj2ffPchiZzw9/W66egeoauggwm7jkiVzyY04zDUXrxx84b6yagFb97Vx77r3uePlWu7fsAebTXB7FH9eW8Q5C1MA6Bvw0NzZQ0J0BNERdr76cAV3vb6bS5akkZ4YTXefmxseqyTMbuOD1mPc9C9t5elZMQ4euW45RRnaf3n18vncu/595idGc80ZLhJnRPhpOdo7wPuHuqhp7OTf2xr4+Qvvceeru1jmSsSup2vDrkMUpM/kgbWn0+/2cMHvN3L9g+U8ct1yfvrcTp6uPMDyzERuOj+dFQuSqN22hbPOOgOA/3qonMfL67nhYzlEhtsZcHt4oryerOQYdjV3cf/6PXz7Yznsbu7iP9sP8OWV2hL396x7n1XZyazOTx31OfKlcn871Qc6uKp49GXnlVLc+EQlOxs7eGDt6Zy9cPhndVeT9p+8/t4hvnCGi6+fm0V8VDjnL5rD1X/ewuV3v0l3v5uSzCTuuXop72x5g+WZWqb+zv42HnijjgsWz6FwXgJ9Ax5+/XINj761nyPdWilWBJZlJJIcFzns9e94uZaXdzTx8o4mzlmYzPdWL+Ro3wDb97dTdaCDA+3dNHX0cPBID8f6tBKGI8zGuppm/vP1UuYnzaD9WB8/fW4np82byT+vX4HNJvT0u+lze3CE2Yiw2xDR7nO/24MIOMLsw6YnKiKKOfFRLHbO5OqS+fz7nQbuebl6MP2RkZG0tLQMadT39haLjBxepxFCxbi8DWSJiAvNqFzJ+NYcnzbUNGrGpe7wUfa3HiM9MXpImIb2bj7+uw2DDzxAdISdn1ySxwxHGN9+rJLdh7pYmBoHQFNHLy1H+8ifGzcYPln/4l2zbB7n3lHGr1+qZVX27MEHrG/Aw7bmAVYMeIgIs+HxKO58dRdZyTH89solFN/+Kv+saBg0LvtajnLZ3W8QFa6lY82yeTzy9n5++FQVV/5pMw1txwDh0S+VkJM6/FL1Bekz+edXVrD2L29x7V/9S5VpM6O4+eOncsWy9MGvZl8K5yVwrM/NHS/XEhcZzo8uyvV7WcpqmslIiuY3VyzhujMz+dVLNRzu6iUyzE5CdATf+kgWnymez+xYB2VlZX7niginZyRyekYiVQ1HuPPVXTR39vKbK5bgmjVjMFxEmA1nwvH7dcsFubz+3iF+8swO7vtcET9/4T32HD7Kw18spmRBEtUHOnh7byur81OZEx81eF7u3Dh+d+Vpw/5HADMcYSx2zmSxcyafKkpnx4EO/vJGHTsbOxC0dF9e6OTHl+QRHaG94n+4qpA1f9rMyl+8zoBHK1H911mnDBrhWp/4P1s8n+febeT5qoN84jQn63cdorGjh1svLuQ/lQf588Y61q7I4Pev7SIyzM51Z7qIjQznjd2Huelf20mNjyQhOpwwu43apk7W1Rxi/a5DFLsSh5Rsfv/abl57r4mLCuYQO8x99XLPuj08s/0g312dM6JhAchKieXG8xZy43n+c2edkhzDE19ZwZf/vpVTU+O4/ROLiAjz/7j53uqFvLazme8+sZ17PruUGx7bRmX9ES5cPIeV2bNJiI7gur9t5cUdTVy93G+lZUB7Z5/ZfoBrz3CRGu/gzld3c95vj1cFzopxMC8xipzUWFZmz2axM55iVxIDbsVFd23ky38v51//tYKfv1BDe3c/f7900eD9iQy3Exnub0DsNsFuG96oDIcjzM4Vp88j5eiewXfI6XRSX1/PoUOHhoT3jnOZKCFhXJRSAyLyNeBFwA48oK/hPmmcaM8Jo9Q0djIrJoLDXX1s3H2YNcv8v+jcHsW3/7GNAbeHez5biCPMjtujyE+LJzU+kvf1BuHt9UcGjUtVwxEAFjnjh1wvIszGN8/N5juPV/LSjibOy0tFKcXNT77L4xW9vHjwDe74dAHvH+piV3MXd645jbjIcFbnp/JM5QF+eGEujjAbP3yqGrsIz37jTObO1DLKq5fPZ3ZMBN94dBsJ0eE8fN1yFgxT3eVLxqwZPP21UjbvaaHf7WHArYiLCmNl1uxhq7t8+fo5p9B+rJ8H3qhjmSuRjy+aA2iNvpv2tHDl6dp/mTs3jgeuOX3EeEa71/lp8dz3uTFrBACYOzOKb5ybxc9feI+fPb+Tv765l2tWZLDilFmDceWnDb0n4yV3bhy/1EukI7HMlcitF+fx1zfquP0Tiwa/1r34ai5ZkETmrBk8tPkDPnGak3+8vZ9ZMRGcszCFU5Jjeb7qIDf9611e3dnEdWdmkhTjAOB3Vy7hgjs3cukf3vCL2xFmI3FGBE++08CtF+cNfmV7PIq397biUVDxQTursodOtOvxKP7y5l5+8eJ7XLh4Dl9ZtWBC/xFoHyjPfP3MEXXHRobz08sWcc1f3uZjv1nHDEcY93y2kNX52nOklCJz1gxerGoc1rjcXbabcLuNr5y1gNmxDi49LY2ntx3AmRDNkvSZpMaPXAr4/ZrTuOYvb7H2gbd4e28bXyx1kevzMTiZ+GoODw+f8DiWsQgJ4wKglHoOeG6q4l+1atVURT2IUorapk7Oy0ulrOYQG3cNNS73rn+fLXWt/OpTBYMPvS+upBnEOMJ4t/4Iny7SagqrDhxBBE6dM/zDeumSufzx9d3c8VItHz01hYe27OPx8nouKpjLpvdbuPiujcRHRbBg9gwu0DPsywudPPlOA6/sbMIuwrraQ/zgwtxBw+Jldf4cXvxWHLGRYczSM6GxiI8K57w841UrXkSEWy44lZd3NvLXN/cOGpctda309HtYlTPmLOHA5N7rL5S6eLx8P/eu20PmrBl8b/WHNwPv1cvnD5spgr9mEeGq4nnc9uxONu46zKs7m7m21EVEmI1TkmO4rNDJE+X1RIXbuW5l5uB5mbNjePYbpVQf6KDf7aFvwMPcmVEscyWyvvYQX/p7OZX7j7DMpZV2a5s7B0vfb9W1DDEuDe3dfOexSjbtaeHchcn88pMFk/6RF3ivz8pJ5gulLt5r7OB/L1vsV3MgIqzOT+Xe9XtoO9pHgk91ZUN7N/+qaOAzxfOYHas958mxkXzxzEyMsDJ7Njeet5Cfv/Aec+Ij+dZHsydB3fAEIy+D0OmKPOV4eyhNBLfHWKeJQ129tB3rJyc1ltKsWbzx/mG/c7fXt3PHS7VcsHgOl+uNzYHYbEJ+Whzb9dIKQFVDBwtmxwxWkQQSZrfxrY9mU9PUyU+e3cGP/7ODcxYmc+2pwkv/byUfzU3hcFcv/++j2dj1YnrJgiS9XWUfP/7PDk6dE8fakuEzLtesGYYNy4liswmfLZ7PW3Wtg1WMZTXNOMJslAR8rY/EidzrQCLCbPz0E4uYlxjNHVcsISrCeDVGMAnUfHmhk4gwG9949B0GPGrwQwXgm+dm4QizsXZFxpD7mjk7hosK5nJZoZMrl81jZfZsIsPtFLuSEIE33z/elvhWXSsAc+IjB/e9bNvfzurfrGd7fTs/v3wR968tmpL/brh7/YMLc3noi8uHrZI+P38Obo/ilZ3+Y0XuW/c+AF86gZLV9asy+f75C/nDZwqJcUzdd/9kPt+jYRkXg+Tl5U3ovBerGzntf14yNF7CmxnmpMRyZtYs2o/1D1ZpHesb4JuPbiM51sFPL1006hfcYudMdh483nOpquGIX3vLcFy4aA45KbH85Y29pCdqbROL8vNJnBHBH64qZNP3z/HrJWO3CZ84LY3Ne1pp7Ojhtkvzx6y2ChafKkonIszGg5u1ZSfW1RxieWbSkDrrkZjovR6J5ZlJrP/u2SxJnzmp8U4mgZoTZkRw4eI5tB7to2h+AqckH6/OTE+MZsN3z+Y7HzP+dR0fHU7+3HjefP94V/Utda3MjY/kwsVzqNx/hJ7+411o/7yxjjC78Pw3V3LF6fOmrFp6vPc6Py2OtJlRvFB1vKvyoc5eHn17P5cVppEWUHIfDyLCl1ctoHBewoTjMMJkP98jMT1yg5OA2trasQMNw7v1R+joGeDOV3eNGXbQuKTGcoZeL+/tNXbbszvZ23KUO65Y4tfldDgWpcXTN+ChtqmTQ529NHb0jFm3b7MJP7gwl4Wpsdx39VLio8IHNYuIX4Ozl8uXOhGBK09PZ+n8qX0hxkOinjH+q6KeHQc62HP4KGcZrBKDid/rk5nhNHur0IbryZUcFznuj4kVC5LY9kE73X3awMe36lpZ5kpkmSuJPreHyv3tgPYh9cqOJj6+aA7zkoaWHiaT8d5rb9XYhl2H6eodoHfAzdcfqcDtUVx/AqWWYBKs59syLgYJHBBllANHtDEJ/6yoH3b0tS+1TVpjflKMg1kxDnLnxLFh1yFe3dnEw1s+4EtnZg5piB2OxXrD/bsNR6g+oJV88uaO3XBcmjWLF761kqwUrTfXWJoXzI7hqa+ewa0XB+dLaDxcvXw+R/vc3PiEVgUwVjdrXyZ6r09mhtN82rwE1t94Np84bfgq2PGyfIFmRMr3tVF3+CiHOnspzkzi9Aztw8RbNfbKzma6+91cVDDyeJLJYiL3enV+Kn1uD68e1odhAAAgAElEQVTubOLb/6hk8x6tDXS4sVnTkWA93yHToD9dOdDeTebsGRxs7+E3r+zi92tG7mJa09RFdsrxbrpnZs3igTfq2N28nVPnxPFtg9UQ8xKjiYsMY3v9kcFpQ6aq58li5/Ss6lmSPpP8tDiqGjqYlxhNxhR/AYcqk1lyOD0jkTCbsGnPYdL1LtvLXInMjI5gYWosb+3VjMt/Kg+QEudgWcb0nFRy6bwEZsc6uPnJKrp6B7j546dy6SQZ4FDCKrlMMQeP9JA3N57Pn5HBfyoPsPPg8NMoeDyKXU2dAcZlNv1uRUfPAL+9YsmIA6UCEREWO2fybkM7VQ1HmJ+kjdI2EyIyWK1zVs7soHUltxiZGEcYi51au8tbda3MiokgUx8ntMyVSPm+NlqP9rGu5hAXLp47ZCDsdMFmE87LS6Grd4DrznT59ZizOI5lXKYQj0dxsL2HufGRfHnlAmIjw7jj5eHrO+vbujnW5/YbYFiUkcCC2TP44YW5Iw48HIlFznjeO9jJOx+0T8pYipORiwvSuLhg7pijvy2Cx4oFs9hef4QNuw+zzJU4aPSXuRI51ufm1y/V0Of2cHEQqsROhG99JJtffaqA759/6tiBTYplXAySmTn+r5PDR3vpc2t9/eOjw7nuzExe3tHE1x95Z0jvMe+0L74ll8hwO6/ecBafHWFswmgsTotnwKO0xnwD7S3DMRHN04moCDt3rjltcDCpUU523RMhWJpLFiRpE4529vpVe3n3H3nrA+YnRQ+2G041E9U9K8bBJ5c6p23pajSCda+tNheDxMaOr+QAcLBdW2jHO7Dw+lUL6Hd7+NOGPbxY1cjaFfP5znk5OMLsg1NiZ6dMTqOg72j8/LSJtbdMRHMoYEbdwdK8dH4CEXYbfW4PxT6dU5LjIslIimZvyzEuWjw3aNWY1r2eOqySi0FaW1vHDhTAAX2Nizn6tA8RYTZu+FgOZd85m0tPm8ufNtTxrUe34fYoaho7SZsZNer8SuMhbWYUCXqXZSM9xYZjIppDATPqDpbmyHA7hfNnEhcZRk6KfybnHbl/8ZLgVYlZ93rqsEouBklJSRn3OQeOaCWXwIFVqfGR/OKTBeSkxvGTZ3bww6eqqGnsHHe7ymiICEvSZ7KruWvIrLpGmYjmUMCMuoOp+ZYLcjnc1TukSum6MzPJTon1qxqeaqx7PXVYJReDjGftFy8H2ruJDLcxc4RBj18odXH9qgU8tOUDagJ6ik0G/3NJPvddbWySxeGYiOZQwIy6g6k5Py1+2HFHWSmxhufimiysez11WCWXKeTgkW7mzowatf74e6tzONTZyz8r6lk4iSUXYNi5kSwsLCyCgWVcppCG9h7mDjNtii8iwv9evoizcmbz0VzzFdEtLCxCE6tabAo52N7N3Jljr+QWbrdxUcFcwxMrWlhYWEx3LONikLCw8RXy+gY8HOrqHbK+ycnEeDWHCmbUbUbNYE7dwdIsShlbayTUKCoqUlPZsLW/9Rhn/uJ1fnH5Yj59evrYJ1hYWFicBIhIuVJqzJ5CVsnFIBUVFeMK36CPcTmZSy7j1RwqmFG3GTWDOXUHS7NlXAyyZMmScYU/qE+1P8dAm8t0ZbyaQwUz6jajZjCn7mBptoyLQaqrq8cV/oB36pcxeotNZ8arOVQwo24zagZz6g6WZsu4GKSlpWXsQD40tHeTEB0+bddMN8J4NYcKZtRtRs1gTt3B0mwZlylC64Z88pZaLCwsLE4Ey7hMEQfaeyzjYmFhYVos4zJFHDjSzdz4k7cx38LCwuJEsIyLQbKysgyH7ezpp7Nn4KQvuYxHcyhhRt1m1Azm1B0szZZxMYjD4TAc9qA+1f6ck9y4jEdzKGFG3WbUDObUHSzNlnExSFdXl+Gw3gGUaSfxGBcYn+ZQwoy6zagZzKk7WJqnnXERkVtFpEFEtunbx338vi8iu0WkRkTO83FfrbvtFpGbpiJdiYmJYwfS8S5vPOckHuMC49McSphRtxk1gzl1B0vztDMuOr9RSi3Rt+cARCQXuBLIA1YDfxQRu4jYgT8A5wO5wBo97KQynikTDnX2ApAce3IXuc04NQaYU7cZNYM5dQdL88k0JeglwKNKqV6gTkR2A8t0v91KqT0AIvKoHnbHh5NM6O53ExFmI8w+XW23hYWFxdQyXY3L10Tkc8BW4AalVBuQBmz2CVOvuwHsD3AvHi5SEbkV+BFAcnIyZWVlfv6pqam4XC5qa2vJy8tj/fr1g35dXV2UlZVRUlJCXV0dTqeTpqYm9u/f7xdHWloaXd29RNhgYGCAjRs3DklHaWkptbW1uFwu6uvraWho8PNPT08nJSWF+vp6XC4XmzZtGhLHypUrqa6uJjs7m7q6OhobG/38MzIySExMpKmpCafTyZYtWwL/C1atWkVlZSV5eXnU1tbS3NzsF6avr4+2tjZaW1tJSUkZsjxqWFgYpaWlVFRUsGTJEqqrq4eM/s3KysLhcNDV1UViYuKQryaHw0FJSQnl5eUsXbqUyspK2tra/MLk5ORgt9vp6ekhNjaWyspKP/+oqCiKi4sH46ioqKCjo8MvTG5uLm63G4/Hg8PhoKqqys8/JiaGoqIiysvLAW0p2MC66fz8fHp7e7HZbNjtdnbs8P9+iYuLo7CwcDAdW7Zsobu72y9MQUEBnZ2dREZG4na7qamp8fNPSEigoKBgMI5NmzbR29vrF6awsJDW1lZiYmLo7e1l165dfv5JSUnk5eWxbds2CgsL2bhxIwMDA35hioqKaGpqIjExkc7OzsHn20tycjLZ2dlUV1dTUFDAunXrCJxBvbi4mPr6elJSUmhtbWXv3r1+/qO9T16MvE9Op5O6ujqys7Mn/X3y6g7G+5SZmUlsbOyH/j719/fT3Nx8Qu+TIZRSQd+AV4CqYbZLgBTAjlZldzvwgH7OXcBnfeL4M/BJfbvfx/1q4K6x0rB06VI1Hl5//XXDYb/7eKVadvvL44p/OjIezaGEGXWbUbNS5tR9opqBrcpAPv+hlFyUUh8xEk5E/gQ8ox82AL4Lozh1N0ZxnzTG032vZ8BNVAisKmnGbppgTt1m1Azm1G3arsgiMsfn8BNoJRqAp4ErRcQhIi4gC3gLeBvIEhGXiESgNfo/PdnpKikpMRy2u88dEksWj0dzKGFG3WbUDObUHSzN0864AL8QkXdFZDtwNvD/AJRS1cBjaA31LwBfVUq5lVIDwNeAF4GdwGN62EnFWxdvhO5+90k9G7KX8WgOJcyo24yawZy6g6XZWuZ4CvjUPW8SZrPxyJeWT0n8FhYWFh8W1jLHk0xgj4rR6On3hETJZTyaQwkz6jajZjCn7mBptoyLQQK7841Gd39oNOiPR3MoYUbdZtQM5tQdLM2WcZkCQqVB38LCwmKiWMZlCujpdxMZbv21FhYW5sXKAaeAnhCpFrOwsLCYKJZxMUhOTo6hcEqpkOmKbFRzqGFG3WbUDObUHSzNlnExiN1uzFj0uT14FCHR5mJUc6hhRt1m1Azm1B0szZZxMUhPT4+xcH0eIDSMi1HNoYYZdZtRM5hTd7A0W8bFILGxsYbCdfe7AUKizcWo5lDDjLrNqBnMqTtYmi3jYhCjA496vMYl4uT/a804wAzMqduMmsGcuq1BlCcpoVRysbCwsJgoYxoX0fisiPxQP54nIsvGOs+seI2LwzIuFhYWJsZIyeWPQAmwRj/uRFuz3mIYevqskouFhYWFkcXCipVShSLyDoBSqk1fN8VUREVFGQrXMxA6xsWo5lDDjLrNqBnMqTtYmo2UXPpFxA4oABGZDXimNFXTkOLiYkPhuvWuyKEwiNKo5lDDjLrNqBnMqTtYmo0YlzuBJ4FkEbkd2Aj8dEpTNQ0xusBOKDXom3EhJTCnbjNqBnPqDpbmMavFlFIPiUg5cC4gwKVKqZ1TnrJpxtKlSw2FO96gf/J3xDOqOdQwo24zagZz6g6W5hFzQBFJ9G5AM/AI8DDQpLuZioqKCkPhekOo5GJUc6hhRt1m1Azm1B0szaOVXMrR2lkEmAe06fszgQ8A15SnbhrR0dFhKFy33lssFKZ/Mao51DCjbjNqBnPqDpbmEUsuSimXUioTeAW4SCk1SymVBFwIvBSU1J2EdPe7CbcL4faTv1rMwsLCYqIYyQGXK6We8x4opZ4HVkxdkk5uuvvdRIad/KUWCwsLixPByDiXAyJyC/CgfvwZ4MDUJenkpqffTWQIdEO2sLCwOBGMlFzWALPRuiM/CSRzfLS+acjNzTUUrqffExKN+WBcc6hhRt1m1Azm1B0szUa6IrcC3xSRWO1QdU19sqYfbrfbULjuvtBZ4tio5lDDjLrNqBnMqTtYmo1MXLlIn/qlCqgWkXIRyZ/6pE0vPB5jkxJ097uJDIExLmBcc6hhRt1m1Azm1B0szUZywXuBbyul5iul5gM3APdNbbKmHw6Hw1A4zbiERsnFqOZQw4y6zagZzKk7WJqNGJcZSqnXvQdKqTJgxpSlaJpSVVVlKFxvvzsk5hUD45pDDTPqNqNmMKfuYGk2Ylz2iMgPRCRD324B9pzIRUXkUyJSLSIeESkK8Pu+iOwWkRoROc/HfbXutltEbvJxd4nIFt39Hx/2jM3d/aHT5mJhYWExUYwYl2vReov9S99m6W4nQhVwGbDe11FEcoErgTxgNfBHEbHrszL/ATgfyAXW6GEBfg78Ril1CtosAl84wbSdEJZxsbCwsDDWW6wN+AaAnsnPUEqd0PwB3okvRSTQ6xLgUaVUL1AnIrsB76qXu5VSe/TzHgUuEZGdwDnAVXqY/wNuBe4+kfSdCN19HmsVSgsLC9MzpnERkYeB6wE38DYQJyK/U0r9cgrSkwZs9jmu190A9ge4FwNJQLtSamCY8EMQkVuBHwEkJydTVlbm55+amorL5aK2tpa8vDzWrz9esDp27BhlZWWUlJRQV1eH0+mkqamJ/fv3+8VxrLefcFHs2LGD7OxsNm7cOCQdpaWl1NbW4nK5qK+vp6Ghwc8/PT2dlJQU6uvrcblcbNq0aUgcK1eupLq6muzsbOrq6mhsbPTzz8jIIDExkaamJpxOJ1u2bAn8L1i1ahWVlZXk5eVRW1tLc3PzkDBtbW20traSkpLC1q1b/fzDwsIoLS2loqKCJUuWUF1dTUtLi1+YrKwsHA4HXV1dJCYmDpk0z+FwUFJSQnl5OUuXLqWyspK2tja/MDk5Odjtdnp6eoiNjaWystLPPyoqiuLi4sE4KioqhsyflJubi9vtxuPx4HA4htQ7x8TEUFRURHl5OTExMWzdupWuLv9e9/n5+fT29mKz2bDb7ezYscPPPy4ujsLCwsF0bNmyhe7ubr8wBQUFdHZ2EhkZidvtpqamxs8/ISGBgoKCwTg2bdpEb2+vX5jCwkJaW1uJiYmht7eXXbt2+fknJSWRl5fHtm3bKCwsZOPGjQwMDPiFKSoqoqmpicTERDo7Owefby/JyclkZ2dTXV1NQUEB69atQynlF0dxcTH19fWkpKTQ2trK3r17/fxHe5+8jPU+paWl4XQ6qaurm5L3yas7GO9TZmYmsbGxH/r7ZLfbaW5uPqH3yQgS+MAMCSCyTSm1REQ+AxQCNwHlSqnFY5z3CpA6jNfNSqmn9DBlwHeUUlv147uAzUqpB/XjPwPP6+etVkp9UXe/Gs243KqHP0V3TweeV0qN2VW6qKhIBd7cyeCU/36OL6/K5MbzFk563BYWFhYfNiJSrpQqGiuckTaXcBEJBy4FnlZK9aOvSjkaSqmPKKXyh9meGuW0BiDd59ipu43k3gLMFJGwAPdJx8gCO/1uDwMeFTJtLmZcSAnMqduMmsGcuoOl2eg4l71o3Y/Xi8h8YKrmbH4auFJEHCLiArKAt9Cq47L0nmERaI3+Tyut2PU68En9/LXAaMZrwhhZYMe7UFiojHMx40JKYE7dZtQM5tT9oS8W5kUpdadSKk0p9XGlsQ84+0QuKiKfEJF6oAR4VkRe1K9VDTwG7ABeAL6qlHLrbSpfA14EdgKP6WEBvgd8W2/8TwL+fCJpGwkjVWg9IWZcpqLa8GTAjLrNqBnMqTtYmkds0BeRzyqlHhSRb48Q5I6JXlQp5Z0Eczi/24Hbh3F/DnhuGPc9HO9RNmUENu4OR0+fNq1CqFSLGdEciphRtxk1gzl1B0vzaL3FvKPwY4ORkFDAWy0WKiP0LSwsLCbKiMZFKXWv/vvj4CXn5OZ4m0toTFxpYWFhMVGMzIqcKSL/EZFDItIsIk+JSGYwEney0d0XWm0uFhYWFhPFyCf2w2iN7HOAucDjwCNTmajpSH7+2KsM9Azo1WIhYlyMaA5FzKjbjJrBnLqDpdmIcYlWSv1dKTWgbw8CkVOdsOlG4Cjp4ejpC602FyOaQxEz6jajZjCn7mBpHnP6F+B5fRbiR9EGT14BPCciiTC4UmXIY7ONbYcHG/RDpORiRHMoYkbdZtQM5tQdLM1GjMun9d8vB7hfiWZsTNH+YrePbTBCbRClEc2hiBl1m1EzmFN3sDQbGUTpGmUzhWEBhkxSCNDQ3k1Vw5HB455+bZxLqBiX4TSbATPqNqNmMKfuYGk20lssWkRuEZH79OMsEblw6pM2/fndK7Vc/+DxeXp6QqxazMLCwmKiGKl8+wvQB6zQjxuA26YsRScRXb0DHDzSw4BbK7F097mx24Rw+5B1aiwsLCxMhRHjskAp9QugH0ApdQywck+gb8CD26M43NUHaG0ukWG24RZBs7CwsDAVRoxLn4hEoU+zLyILANP134uLixvi1juglVgaO3oArVosVLohw/CazYAZdZtRM5hTd7A0GzEuP0KboThdRB4CXgW+O6WpmoYUFhYOcevzGpcj2mqD3f3ukGnMh+E1mwEz6jajZjCn7mBpNtJb7GXgMuAatJH5RUqpsqlN1vRjuAV2BksuR3xKLiFkXMy4kBKYU7cZNYM5dQdLs5FxLiilWoBnpzgt05rhFtjxllwO6tVi3X2hVXIx40JKYE7dZtQM5tQ9bRYLs9DYsmXLELc+t3/JpTvESi7DaTYDZtRtRs1gTt3B0jyicdGXGbbQ6e7uHuLWN6RazENkCDXoD6fZDJhRtxk1gzl1B0vzaCWXJwBE5NWgpOQkpG+43mLWWi4WFhYWo7a52ETkv4Hs4ZY6VkpNeJnjUMG3WkwpFXK9xSwsLCwmymif2VcCbjQDFDvMZnr6BjzYbULvgIf2Y/1094VWm4uFhYXFRBltmeMa4Ocisl0p9XwQ0zQtKSgoGOLWN+AhbWYUH7Qeo7Gjh54QK7kMp9kMmFG3GTWDOXUHS7ORBoI3ReQOEdmqb78WkfgpT9k0o7Oz0+9YKUWf28P8pGhAqxrr6feE1Aj9QM1mwYy6zagZzKk7WJqNGJcHgE60dV0+DXSgTWZpKiIj/Rff9La3pCdqxqW+vZs+tyekqsUCNZsFM+o2o2Ywp+5gaTYyiHKBUupyn+Mfi8i2qUrQdMXtdvsde3uKOROiEIG9h48CEBlCvcUCNZsFM+o2o2Ywp+5gaTaSE3aLSKn3QETOAEzXObympsbv2GtcZkSEMTvGMWhcQqnkEqjZLJhRtxk1gzl1B0uzkZLL9cDffNpZ2oC1U5ekkwNvtVhEmI058ZHUtXhLLqFjXCwsLCwmipGJKyuVUgXAYmCxUuo0pdT2E7moiHxKRKpFxCMiRT7uGSLSLSLb9O0eH7+lIvKuiOwWkTtFXzRFRBJF5GUR2aX/JpxI2oziLblE2G2kxEXyQcsxgJBq0LewsLCYKIYbCJRSHUqpjkm6bhXaTMvrh/F7Xym1RN+u93G/G7gOyNK31br7TcCrSqkstOUAbpqkNI7KoHHRSy4DHgVAZJhlXCwsLCw+lNZnpdROfRyNIURkDhCnlNqslFLA34BLde9LgP/T9//Px31SSUjwLxD1+hiXlPjjvS9CqeQSqNksmFG3GTWDOXUHS/N07NrkEpF3RGSdiJypu6UB9T5h6nU3gBSl1EF9vxFImYpEBQ48Cmxz8RJKbS5mHGAG5tRtRs1gTt3B0jxmg76IfAp4QSnVKSK3AIXAbUqpijHOewVIHcbrZqXUUyOcdhCYp5RqEZGlwL9FJG+sNHpRSikRUaOk6Va0lTVJTk6mrKzMzz81NRWXy0VtbS15eXmsX3+81u7YsWNER0dTUlJCXV0dR9wxAOysehfxiePQwQa6kyOpq6sjOzubjRs3DklHaWkptbW1uFwu6uvraWho8PNPT08nJSWF+vp6XC4XmzZtGhLHypUrqa6uJjs7m7q6OhobG/38MzIySExMpKmpCafTOWSabRFh1apVVFZWkpeXR21tLc3NzX5hbDYbixYtorW1lZSUFLZu3ernHxYWRmlpKRUVFSxZsoTq6mpaWlr8wmRlZeFwOOjq6iIxMZGKCv/HxuFwUFJSQnl5OUuXLqWyspK2tja/MDk5Odjtdnp6eoiNjaWystLPPyoqiuLi4sE4Kioq6Ojwr8HNzc3F7Xbj8XhwOBxUVVX5+cfExFBUVDS4kJJSiq6uLr8w+fn59Pb2YrPZsNvt7Nixw88/Li6OwsLCwXRs2bJlyAy0BQUFdHZ2EhkZidvtHtJzJyEhgYKCgsE4Nm3aRG+v/6rihYWFtLa2EhMTQ29vL7t27fLzT0pKIi8vj23btlFYWMjGjRsZGBjwC1NUVERTUxOJiYl0dnZSVVVFdHT0oH9ycjLZ2dlUV1dTUFDAunXr0CoNjlNcXEx9fT0pKSm0trayd+9eP//R3icv3vfJ6XTS1NTE/v37/fzT0tJwOp1T9j553+tgvE+ZmZnExsZ+6O9TWFgY2dnZJ/Q+GUECH5ghAbTpXxbr3ZFvA34J/FApVTzuqw2Nuwz4jlJq62j+QAPwulJqoe6+BjhLKfVlEanR9w/q1WdlSqmcsa5dVFSkAm/ueFhfe4jPPfAWT1xfQlKMg7N/VQbAazesInN2zITjtbCwsJjOiEi5UqporHBGqsW8I24uAO5TSj0LRJxI4kZCRGaLiF3fz0RruN+jV3t1iMhyvZfY5wBv6edpjneNXuvjPqkElh5821xS40KzzWW4EpMZMKNuM2oGc+oOlmYjxqVBRO4FrgCeExGHwfNGREQ+ISL1QAnwrIi8qHutBLbrMwA8AVyvlGrV/f4LuB/YDbwPeCfT/F/goyKyC/iIfjzpBFZN+PYWi4qwEx8VDoTWIMpAzWbBjLrNqBnMqTtYmo0Movw0WrffXyml2vWqpxtP5KJKqSeBJ4dx/yfwzxHO2QrkD+PeApx7IumZCH36FAoRds3OpsZFcqS7P6Qa9C0sLCwmymjLHCeKSCIQCZQBLfpxLzDxxooQwbfkApAaH4kIOMKmYwc8CwsLi+AyWsmlHFCAAPPQpn0RYCbwAeCa8tRNYwKNy5z4SCLD7OgTB1hYWFiYmtEWC3MBiMifgCeVUs/px+czRQMVpzOFhYV+x94GfYddqwb77PL5LHbODHq6ppJAzWbBjLrNqBnMqTtYmo3U4Sz3GhYAfVXKFVOXpOlJa2ur37HvIEqA/LR4riqeF/R0TSWBms2CGXWbUTOYU3ewNBtp0D+gD558UD/+DHBg6pI0PYmJ8R+7ElgtFooEajYLZtRtRs1gTt3B0mwkZ1wDzEbr3fUvfX/NVCZqOjJcV2S7TbDbQreNxYzdNMGcus2oGcypO1iaRzUu+oDG/1ZKfVOfar9QKfUtn7EnpiFwio2+Ac9gN+RQJVCzWTCjbjNqBnPqDpbmUXNHpZQbKB0tjFnpc3twhNCSxhYWFhaTiZE2l3dE5GngceCo11Ep9a8pS9VJgBlKLhYWFhYTxYhxiQRagHN83BRa+4tp6RvwhHRjvoWFhcWJMKZxUUp9PhgJme4kJSX5Hfe6Q9+4BGo2C2bUbUbNYE7dwdJsZD2XSOALQB5aKQYApdS1U5iuaUdenv+yMmaoFgvUbBbMqNuMmsGcuoOl2Uju+He0Rb/OA9YBTqBzKhM1Hdm2bZvfcd+AJ+TnEQvUbBbMqNuMmsGcuoOl2chiYe8opU7zWTQsHNiglFoelBROESe6WNia+zYz4PHw+PWmm6zAwsLCxEzmYmH9+m+7iOQD8UDyiSTuZCRwedU+E7S5DLekrBkwo24zagZz6g6WZiO9xe4TkQTgB2irPsbo+6YicA3yvgEPcZFG/r6Tl0DNZsGMus2oGcypO1iajfQWu1/fXQdkTm1yTh6srsgWFhYWI2Okt9j7wGZgA1pbS/WUp+okoHfATUSYteqkhYWFxXAY+fTOBe4FkoBfisj7IjJkiWKzYYauyBYWFhYTxUju6EZr1HcDHqBZ30xFUZF/5wgzNOgHajYLZtRtRs1gTt3B0mwkd+wAfgvUAWuVUiVKqS9PbbKmH01NTX7HvSYY5xKo2SyYUbcZNYM5dQdLs9H1XNYD/wU8KiI/FpFzpzZZ04/ExES/YzM06AdqNgtm1G1GzWBO3cHSPGbuqJR6Sil1I/Bl4DngGuCZKU7XtKOz8/ikBEoprVosxNtcfDWbCTPqNqNmMKfuYGkeM3cUkX+KyG7gd0A08DkgYaoTNt3Ys2fP4P6AR6FUaC9xDP6azYQZdZtRM5hTd7A0GxkF+DPgHX3hMAu0KjEIfeNiYWFhMVGM5I47gO+LyH0AIpIlIhdObbKmN4PGJcSrxSwsLCwmipHc8S9AH+CdobEBuG3KUnQS0OfWjIu1zLGFhYXF8BjJHRcopX6BPoGlUuoYICdyURH5pYi8JyLbReRJEZnp4/d9EdktIjUicp6P+2rdbbeI3OTj7hKRLbr7P0Qk4kTSNhLJycfn6jRLycVXs5kwo24zagZz6g6WZiO5Y5+IRKEtbYyILAB6T/C6LwP5SqnFQC3wfT3uXOBKtIXJVgN/FBG7iNiBPwDno80YsEYPC/Bz4DdKqVOANrSFzSad7Ozswf1ek7S5+Go2E2bUbUbNYE7dwdJsJHf8EeBCv8sAABOBSURBVPACkC4iDwGvAt89kYsqpV5SSnmn5tyMtgAZwCXAo0qpXqVUHbAbWKZvu5VSe5RSfcCjwCUiIsA5wBP6+f8HXHoiaRuJ6urjU6p5Sy6hPojSV7OZMKNuM2oGc+oOluZRe4vpmfd7wGXAcrTqsG8qpQ5PYhquBf6h76ehGRsv9bobwP4A92K0+c7afQyVb/ghiMitaMaS5ORkysrK/PxTU1NxuVzU1taSl5fH+vXr/fzLysooKSlh1/taV76WQ82UldX4hUlLS8PpdFJXV0d2dvawayeUlpZSW1uLy+Wivr6ehoYGP//09HRSUlKor6/H5XKxadOmIXGsXLmS6upqsrOzqauro7Gx0c8/IyODxMREmpqacDqdbNmyJfC/YNWqVVRWVpKXl0dtbS3Nzf6z+mRmZtLW1kZrayspKSkELq4WFhZGaWkpFRUVLFmyhOrqalpaWvzCZGVl4XA46OrqIjExkYqKCj9/h8NBSUkJ5eXlLF26lMrKStra2vzC5OTkYLfb6enpITY2lsrKSj//qKgoiouLB+OoqKigo6PDL0xubi5utxuPx4PD4aCqqsrPPyYmhqKiosE4tm7dSldXl1+Y/Px8ent7sdls2O12duzY4ecfFxdHYWHhYBxbtmyhu7vbL0xBQQGdnZ1ERkbidrupqfF/fhISEigoKBiMY9OmTfT2+lcUFBYW0traSkxMDL29vezatcvPPykpiby8PLZt20ZhYSEbN24cMs16UVERTU1NJCYm0tnZSVtbm9/7kJycTHZ2NtXV1RQUFLBu3ToCFxYsLi6mvr6elJQUWltb2bt3r5//WO8TQElJCXV1dTidTpqamti/f7+ffzDep7KysqC9T7GxsdPifWpubj6h98kQSqlRN+DdscKMcN4rQNUw2yU+YW4GnuT4iph3AZ/18f8z8El9u9/H/Wo97Cy0Eo3XPR2oMpK+pUuXqvFQVlY2uL9lT4ua/71n1IbaQ+OK42TDV7OZMKNuM2pWypy6T1QzsFUZyGONjHOpEJHTlVJvj9NofWQ0fxG5BrgQOFdPMGg90dJ9gjl1N0ZwbwFmikiY0kovvuEnleNJNM84F1/NZsKMus2oGcypO1iajeSOxcAmfar97SLyrohsP5GLishqtHabi5XW+8zL08CVIuIQEReQBbwFvA1k6T3DItAa/Z/WjdLraCUbgLXAUyeSNiP0ubXxpKFuXCwsLCwmipGSy3ljBxk3dwEO4GWtWYfNSqnrlVLVIvIY2sDNAeCrSp8ZQES+BrwI2IEH1PFFy76HNqHmbcA7aFVpU4pZuiJbWFhYTBQjyxzvm+yLKq3b8Eh+twO3D+P+HNrEmYHue9B6kwUNs3RFtrCwsJgoVu5okOLi/9/e3QfHVZ13HP/+LAdBsAmWi2UiGSwXi9RyI0fWYDQoJgk0Q1ImpB0maRsmlKTtpCUtTdsk0KaTwqTTZMg0DU2HZiakkCahZEhSKG1TiIMpbo2LLGywTBAvIrUcLDHIYKvAypKe/nHvKrvClvVyd9fc5/nM7PjuuXel89OxztE9923T1LKXU5FLM3viMbfHzOAzd7Uy57t3zNDg4ODUspc9l9LMnnjM7TEz+Mxdrcz57h0z1NjYOLXs5ZhLaWZPPOb2mBl85q5W5nz3jhkaGRmZWi7euDLvey6lmT3xmNtjZvCZu1qZ8907Zqj06mMv17lMv+LaC4+5PWYGn7mrlTnfvWOFjI1PIsHiRQu6OXQIIeRWDC7zMDYxyUl1i0iv0QkhhDBNDC7zMDY+mfspsRBCWIjoIWdp5cqVU8uF8UnqF9fVsDbVUZrZE4+5PWYGn7mrlTkGl1lqaWmZWh4bn8z9BZRQntkTj7k9ZgafuauVOf89ZEb6+/unlscmfEyLlWb2xGNuj5nBZ+5qZc5/D5mRtra2qeWx8YncX0AJ5Zk98ZjbY2bwmbtamfPfQ2ak9Cl6Xg7oH+3JgR54zO0xM/jMXa3M+e8hK8DLtFgIIcxX9JDzMDY+6WJaLIQQ5it6yHnwMi0WQgjzFT3kPBRicAkhhBlFDzlLXV1dU8tejrmUZvbEY26PmcFn7mplzn8PmZGBgYGp5bHxSeodHHMpzeyJx9weM4PP3NXKnP8eMiPNzc1Ty16OuZRm9sRjbo+ZwWfuamXOfw+ZkaGhoallL9NipZk98ZjbY2bwmbtamfPfQ2Zk3759U8uFIz5ORS7N7InH3B4zg8/c1cqc/x6yArzsuYQQwnxFDzlHE5PGxKTF4BJCCDOIHnKOxsYnAWJwCSGEGUQPOUtNTU1AyeDi4JhLMbM3HnN7zAw+c1crc/57yIwUT98rTEwAuHhYmMfTNMFnbo+ZwWfuXJ+KLOlGST+W9Kik70s6PS1fLekVSbvS19+XfGajpMckPSXpJklKyxsk3SfpyfTfZZWoc/HCo+Kei4fHHHu8wAx85vaYGXzmzvtFlPcB683srUA/cF3JuqfNbEP6+lhJ+c3AbwNr09clafm1wBYzWwtsSd9nrrW1FfB1zKWY2RuPuT1mBp+5q5W5Jj2kmd1rZuPp24eAGffTJJ0JnGZmD5mZAd8A3p+uvgy4LV2+raQ8U9u2bQOS05DBx+BSzOyNx9weM4PP3NXKvLgq32VmHwHuKHnfIukR4BDwGTN7EGgCBku2GUzLABrN7Ll0+QDQeKxvJOkvgM8CrFixgq1bt5atX7lyJS0tLfT399PW1lb2xLbR0VG2bt3KkrOSR4ROHinw9NNPv+aCpKamJpqbmxkYGKC1tfWoDdnd3U1/fz8tLS0MDg6yf//+svWrVq2isbGRwcFBWlpa2L59+2u+xubNm+nr66O1tZWBgQEOHDhQtn716tU0NDQwNDREc3MzO3bsmP6z4MILL2T37t20tbXR39/P8PBw2TZjY2McPHiQkZERGhsb6enpKVu/ePFiuru76e3tZcOGDfT19fHCCy+UbbN27Vrq6+sZHR2loaGB3t7esvX19fV0dXWxc+dONm7cyO7duzl48GDZNueeey51dXW8+uqrLF26lN27d5etP+WUU9i0adPU1+jt7eXQoUNl26xbt46JiQkmJyepr69nz549ZeuXLFlCZ2cnO3fuBKCnp4fR0dGybdavX0+hUGDRokXU1dWxd+/esvWnnXYaHR0dU/XYsWMHr7zyStk27e3tHD58mJNPPpmJiQmeeOKJsvXLli2jvb196mts376dQqFQtk1HRwcjIyMsWbKEQqHAk08+WbZ++fLltLW1sWvXLjo6Oti2bRvj4+Nl23R2djI0NERDQwOHDx+e+v9dtGLFClpbW+nr66O9vZ0HHniA5O+6n9m0aRODg4M0NjYyMjLCs88+W7Z+pt+noq6uLgYGBmhubmZoaKjqv0/F3NX4fVqzZg1Lly6t+e/TkSNHGB4eXtDv06yYWUVewA+BPUd5XVayzZ8B3weUvq8HlqfLG4F9wGlAJ/DDks+9HbgnXX5x2vc9OJv6bdy40ebi/vvvNzOzhwdesLM/fY898MTwnD7/elTM7I3H3B4zm/nMvdDMQI/Noo+t2J6LmV0803pJvwlcClyUVhgzKwCFdHmnpKeBVmA/5VNnzWkZwJCkM83suXT6rPzPhYx5OuYSQgjzVauzxS4BPgW8z8xeLik/Q1JduryG5MD9M5ZMex2SdH56ltiHgbvSj90NXJkuX1lSXhEFR8dcQghhvmp1zOUrJFNg96VnFD9kyZlhm4EbJB0BJoGPmdlI+pnfA24FTgH+PX0BfB74jqSPAj8BPlCJCnd3dwO+LqIsZvbGY26PmcFn7mplrtXZYueY2SqbdsqxmX3XzNrSsg4z+5eSz/SY2Xoz+3kz+3jJVNoLZnaRma01s4tLBqNM9ff3A6XXueR/cClm9sZjbo+ZwWfuamXOfw+ZkZaWFsDXMZdiZm885vaYGXzmrlbm/PeQGRkcTM6E9nSdSzGzNx5ze8wMPnNXK3P+e8iMFM+d93TMZfr1Al54zO0xM/jMXa3M+e8hM+ZpWiyEEOYresg5Kownd0WOwSWEEI4tesg58jQtFkII8xU95CytWrUKSC6iPKluEen1OblWzOyNx9weM4PP3NXKHIPLLDU2JvfDHBufdDMlVszsjcfcHjODz9zVyuyjl8zA1KnIjgYXj6dpgs/cHjODz9xxKvIJpvQiSi/HWzxeYAY+c3vMDD5zx0WUJ5jiMyDGJiapf4OPH9vRniPjgcfcHjODz9zVyuyjl8yQpz2XEEKYr+gl58jTMZcQQpivE+Exx68rHWcv4/Cr48ffMIQQHIvBZY6ufuc5ta5CCCGc8IrPrnens7PTenp6Zr395OQkixb5mg7zmBl85vaYGXzmXmhmSTvNrPN42/n6qS5AX19fratQdR4zg8/cHjODz9zVyhx7LrNUKBSor6+vYI1OPB4zg8/cHjODz9wLzRx7LhkbGBiodRWqzmNm8JnbY2bwmbtamWNwmaUDBw7UugpV5zEz+MztMTP4zF2tzDG4hBBCyFwMLiGEEDIXg0sIIYTMuT1bTNLzwE/m8JE3Az+tUHVOVB4zg8/cHjODz9wLzXy2mZ1xvI3cDi5zJcnMLP+PnyzhMTP4zO0xM/jMXa3MMS0WQgghczG4hBBCyFwMLrN3fa0rUAMeM4PP3B4zg8/cVckcx1xCCCFkLvZcQgghZC4GlxBCCJmLweU4JF0i6QlJT0m6ttb1qRRJqyTdL2mvpD5J16TlDZLuk/Rk+u+yWtc1a5LqJD0i6Z70fYukHWmb3yHppFrXMWuSTpd0p6QfS3pcUlfe21rSJ9L/23sk3S7p5Dy2taSvSxqWtKek7Khtq8RNaf5HJXVkVY8YXGYgqQ74O+A9wDrg1yWtq22tKmYc+GMzWwecD1ydZr0W2GJma4Et6fu8uQZ4vOT9F4Avmdk5wEHgozWpVWV9GfiBmb0FaCfJn9u2ltQE/AHQaWbrgTrg18hnW98KXDKt7Fht+x5gbfr6HeDmrCoRg8vMzgOeMrNnzGwM+CfgshrXqSLM7Dkz602XD5N0Nk0keW9LN7sNeH9talgZkpqBXwa+lr4X8C7gznSTPGZ+E7AZuAXAzMbM7EVy3tYkj3U/RdJi4I3Ac+Swrc3sP4GRacXHatvLgG9Y4iHgdElnZlGPGFxm1gTsK3k/mJblmqTVwNuAHUCjmT2XrjoANNaoWpXyN8CngMn0/XLgRTMbT9/nsc1bgOeBf0inA78m6VRy3NZmth/4IvC/JIPKS8BO8t/WRcdq24r1cTG4hDKSlgDfBf7QzA6VrrPkvPXcnLsu6VJg2Mx21rouVbYY6ABuNrO3Af/HtCmwHLb1MpK/0ltI7q11Kq+dOnKhWm0bg8vM9gOrSt43p2W5JOkNJAPLt8zse2nxUHE3Of13uFb1q4ALgPdJepZkyvNdJMciTk+nTiCfbT4IDJrZjvT9nSSDTZ7b+mJgwMyeN7MjwPdI2j/vbV10rLatWB8Xg8vMHgbWpmeUnERyAPDuGtepItJjDbcAj5vZX5esuhu4Ml2+Erir2nWrFDO7zsyazWw1Sdv+yMw+BNwPXJ5ulqvMAGZ2ANgn6dy06CJgLzlua5LpsPMlvTH9v17MnOu2LnGstr0b+HB61tj5wEsl02cLElfoH4ek95LMy9cBXzezv6xxlSpCUjfwIPAYPzv+8Kckx12+A5xF8oiCD5jZ9IOFr3uS3gH8iZldKmkNyZ5MA/AIcIWZFWpZv6xJ2kByEsNJwDPAVSR/bOa2rSVdD3yQ5MzIR4DfIjm+kKu2lnQ78A7g54Ah4LPAP3OUtk0H2q+QTBG+DFxlZj2Z1CMGlxBCCFmLabEQQgiZi8ElhBBC5mJwCSGEkLkYXEIIIWQuBpcQQgiZi8ElhCqSdIOkizP4OqNZ1CeESolTkUN4HZI0amZLal2PEI4l9lxCWCBJV0j6H0m7JH01fT7MqKQvpc8P2SLpjHTbWyVdni5/Pn1+zqOSvpiWrZb0o7Rsi6Sz0vIWSdslPSbpc9O+/yclPZx+5vq07FRJ/yppd/r8kg9W96cSvIvBJYQFkPQLJFd9X2BmG4AJ4EMkN0bsMbM24AGSq6RLP7cc+BWgzczeChQHjL8FbkvLvgXclJZ/meRGk79Iclff4td5N8mzOM4DNgAbJW0mueL6p2bWnj6/5AeZhw9hBjG4hLAwFwEbgYcl7UrfryG5hc4d6TbfBLqnfe4l4FXgFkm/SnLrDYAu4Nvp8j+WfO4C4PaS8qJ3p69HgF7gLSSDzWPAL0n6gqS3m9lLC8wZwpwsPv4mIYQZiGRP47qyQunPp21XdnDTzMYlnUcyGF0OfJzkrswzOdoBUgF/ZWZffc2K5JG17wU+J2mLmd1wnK8fQmZizyWEhdkCXC5pBUw9q/xskt+t4t12fwPYVvqh9Lk5bzKzfwM+QfKoYYD/JrlDMyTTaw+my/81rbzoP4CPpF8PSU2SVkh6M/CymX0TuJHklvohVE3suYSwAGa2V9JngHslLQKOAFeTPIDrvHTdMMlxmVJLgbsknUyy9/FHafnvkzwh8pMkT4u8Ki2/Bvi2pE9Tclt4M7s3Pe6zPbnBLaPAFcA5wI2SJtM6/W62yUOYWZyKHEIFxKnCwbuYFgshhJC52HMJIYSQudhzCSGEkLkYXEIIIWQuBpcQQgiZi8ElhBBC5mJwCSGEkLn/B23kxxaSS3JyAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learning_curve(data, 2, 1, title=\"DQNAgent performance on PuckWorld\", \n",
    "               x_name=\"episodes\", y_name=\"rewards of episode\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/20 [00:00<?, ?it/s]/home/qiang/Documents/Codes/c06/agents.py:222: UserWarning: invalid index of a 0-dim tensor. This will be an error in PyTorch 0.5. Use tensor.item() to convert a 0-dim tensor to a Python number\n",
      "  mean_loss = loss.sum().data[0] / self.batch_size\n",
      "  5%|▌         | 1/20 [00:00<00:11,  1.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epsilon:0.00,loss:0.72,episode 100    68 steps,total reward:-29.32  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 10%|█         | 2/20 [00:01<00:09,  1.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epsilon:0.00,loss:0.93,episode 101   134 steps,total reward:-22.30  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 15%|█▌        | 3/20 [00:01<00:07,  2.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epsilon:0.00,loss:0.97,episode 102   115 steps,total reward:-51.25  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 20%|██        | 4/20 [00:01<00:06,  2.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epsilon:0.00,loss:0.87,episode 103   100 steps,total reward:-23.31  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 25%|██▌       | 5/20 [00:01<00:05,  2.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epsilon:0.00,loss:0.86,episode 104    86 steps,total reward:-17.93  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 30%|███       | 6/20 [00:02<00:05,  2.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epsilon:0.00,loss:0.96,episode 105   100 steps,total reward:-19.08  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 35%|███▌      | 7/20 [00:03<00:07,  1.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epsilon:0.00,loss:0.78,episode 106   525 steps,total reward:-74.30  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 40%|████      | 8/20 [00:04<00:07,  1.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epsilon:0.00,loss:0.66,episode 107   295 steps,total reward:-58.67  \n",
      "epsilon:0.00,loss:1.09,episode 108    27 steps,total reward:-6.65   \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 50%|█████     | 10/20 [00:06<00:06,  1.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epsilon:0.00,loss:0.79,episode 109   383 steps,total reward:-89.04  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 55%|█████▌    | 11/20 [00:06<00:05,  1.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epsilon:0.00,loss:0.60,episode 110    87 steps,total reward:-30.22  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 60%|██████    | 12/20 [00:07<00:04,  1.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epsilon:0.00,loss:0.67,episode 111   303 steps,total reward:-63.39  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 65%|██████▌   | 13/20 [00:07<00:04,  1.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epsilon:0.00,loss:0.75,episode 112   111 steps,total reward:-45.54  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 70%|███████   | 14/20 [00:08<00:03,  1.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epsilon:0.00,loss:0.80,episode 113   152 steps,total reward:-22.72  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 75%|███████▌  | 15/20 [00:08<00:02,  1.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epsilon:0.00,loss:0.60,episode 114   259 steps,total reward:-61.19  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 80%|████████  | 16/20 [00:13<00:03,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epsilon:0.00,loss:0.67,episode 115  1423 steps,total reward:-414.71 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 85%|████████▌ | 17/20 [00:15<00:02,  1.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epsilon:0.00,loss:0.70,episode 116   549 steps,total reward:-107.43 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 90%|█████████ | 18/20 [00:15<00:01,  1.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epsilon:0.00,loss:0.67,episode 117    91 steps,total reward:-32.64  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 95%|█████████▌| 19/20 [00:16<00:00,  1.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epsilon:0.00,loss:0.70,episode 118   117 steps,total reward:-52.56  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "100%|██████████| 20/20 [00:17<00:00,  1.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epsilon:0.00,loss:0.67,episode 119   393 steps,total reward:-108.90 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "data = agent.learning(gamma=0.99,          # 衰减引子\n",
    "               epsilon = 0.0001,\n",
    "               decaying_epsilon = False,\n",
    "               alpha = 0.001, \n",
    "               max_episode_num = 20, \n",
    "               display=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
